from spider import Spider
from pysqlite3 import PySqlite3
from pysession import Session
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import re
import time
import sqlite3
import queue
import threading
import requests

class DouBan(PySqlite3):

    def __init__(self, db_name):
        super().__init__(db_name)
        self.create_db()
        self.class_queue = queue.Queue(50)
        self.movie_queue = queue.Queue(50)


    def create_db(self):
        sql = """
            CREATE TABLE class_movie(id integer PRIMARY KEY autoincrement, class_name TEXT, class_url TEXT);
            CREATE TABLE movie(id integer PRIMARY KEY autoincrement, movie_name TEXT,
                                movie_url TEXT, this_class_id integer);
            CREATE TABLE movie_info(id integer PRIMARY KEY autoincrement, driect TEXT, scirptwriiter TEXT,
                                      actor TEXT, genre_type TEXT, state TEXT, language TEXT, data TEXT,
                                      run_time TEXT, IMDb TEXT, this_movie_id integer);
            CREATE TABLE grade(id integer PRIMARY KEY autoincrement,fen TEXT, five TEXT, four TEXT, three TEXT,
                                two TEXT, one TEXT, this_movie_id integer);
            CREATE TABLE content(id integer PRIMARY KEY autoincrement, author TEXT, content TEXT,
                                  time TEXT, content_url TEXT, this_movie_id TEXT)
        """
        try:
            self.script(sql)
            print("数据库创建成功")
        except sqlite3.OperationalError:
            print("数据库已经存在")

    #输入一个分类的url获取到所有的分类链接
    def get_all_class_url(self,url):
        """
        输入一个分类的url获取到所有的分类链接
        :param url:  https://movie.douban.com/explore#!type=movie&tag=最新&page_limit=20&page_start=0
        :return: [{class_name:'',class_url:''},........]
        """
        option = webdriver.ChromeOptions()

        option.add_argument("headless")

        dirver = webdriver.Chrome(chrome_options=option)

        dirver.get(url)

        info = dirver.page_source

        # print(info)

        class_all_url = []

        class_name_regex = '<input type="radio" name="tag" value="(.*?)" />'

        class_re = re.findall(class_name_regex, info)

        # print(class_re)
        for name in class_re[1:]:

            class_info_dict = {}

            class_url = 'https://movie.douban.com/explore#!type=movie&tag={}&page_limit=20&page_start=0'.format(name)

            class_info_dict['class_name'] = name
            class_info_dict['class_url'] = class_url

            class_all_url.append(class_info_dict)

        print(class_all_url)

        return class_all_url

    #输入一个分类的url，获取到所有的电影链接
    def get_one_class_all_movie_url(self,url):
        """
        输入一个分类的url，获取到所有的电影链接
        :param url: https://movie.douban.com/explore#!type=movie&tag=%E7%83%AD%E9%97%A8&sort=time&page_limit=20&page_start=0
        :return:  [{movie_name:'',movie_url:''},{movie_name:'',movie_url:''},...........]
        """
        dirver = webdriver.Chrome()
        dirver.get(url)
        for i in range(5):
            above = dirver.find_element_by_class_name("more")    #定位到悬停元素
            above.send_keys(Keys.ENTER)                   #模拟点击回车键
            time.sleep(3)
        info = dirver.page_source
        # print(info)
        movie_info_regex =    '<div class="cover-wp" data-isnew="false" data-id="(.*?)">[\s\S]+?<img src=".*?" alt="(.*?)" data-x=".*?" data-y=".*?" />[\s\S]+?</div>'
        movie_info = re.findall(movie_info_regex, info)
        # print(movie_info)
        class_name_regex = "https://movie.douban.com/explore#!type=movie&tag=(.*?)&page_limit=20&page_start=0"
        class_name = re.findall(class_name_regex, url)[0].encode('gbk')
        # print(class_name.decode('gbk'))
        all_movie_info = []
        for i in movie_info:
            one_movie_info = {}
            one_movie_info['movie_name'] = i[1]
            one_movie_info['movie_url'] = 'https://movie.douban.com/subject/{}/?tag={}&from=gaia_video'.format(i[0],class_name)
            all_movie_info.append(one_movie_info)
        print(all_movie_info)
        return all_movie_info

    #输入豆瓣电影中一部电影的页面，返回相应的内容
    def get_movie_info(self, url):
        """
        输入豆瓣电影中一部电影的页面，返回相应的内容
        :param url:  https://movie.douban.com/subject/30163509/?tag=热门&from=gaia_video
        :return:  {'driect':[], 'scirptwriter':'', 'actor':[], 'genre_type':'',
                   'state':'', 'language':'', 'data':'', 'run_time':'', 'IMDb':[]}
        """

        direct_regex = '<a href=".*?" rel="v:directedBy">(.*?)</a>'

        scriptwriter_regex = " <span ><span class='pl'>编剧</span>: <span class='attrs'><a href=.*?>(.*?)</a>"

        actor_regex = 'rel="v:starring">(.*?)</a>'

        genre_type_regex = '<span property="v:genre">(.*?)</span>'

        state_regex = '<span class="pl">制片国家/地区:</span> (.*?)<br/>'

        language_regex = '<span class="pl">语言:</span> (.*?)<br/>'

        data_regex = '<span property="v:initialReleaseDate" content="(.*?)">'

        run_time_regex = '<span property="v:runtime" content=".*?">(.*?)</span>'

        IMDb_regex = '<span class="pl">IMDb链接:</span> <a href="(.*?)"'

        info = Spider().get_info(url, driect=direct_regex, scirptwriter=scriptwriter_regex, actor=actor_regex,
                                 genre_type=genre_type_regex, state=state_regex, language=language_regex,
                                 data=data_regex, run_time=run_time_regex, IMDb=IMDb_regex)

        print(info)

        try:

            info['scirptwriter'] = info['scirptwriter'][0]

            info['genre_type'] = info['genre_type'][0]

            info['state'] = info['state'][0]

            info['language'] = info['language'][0]

            info['data'] = info['data'][0]

            info['run_time'] = info['run_time'][0]

            info['IMDb'] = info['IMDb']
        except IndexError:

            pass

        print(info)

        return info

    #输入一部电影的链接，获取到电影的评分
    def get_movie_grade(self, url):
        """
        输入一部电影的链接，获取到电影的评分
        :param url: https://movie.douban.com/subject/30163509/?tag=热门&from=gaia_video
        :return:{'five_star': {'name': '5星', 'fen': '12.5%'}, 'four_star': {'name': '4星', 'fen': '38.2%'}, 'three_star': {'name': '3星', 'fen': '38.4%'}, 'two_star': {'name': '2星', 'fen': '8.7%'}, 'one_star': {'name': '1星', 'fen': '2.1%'}}
        """
        fen_regex = 'property="v:average">(.*?)</strong>'
        five_star_regex = '<div class="item">[\s\S]+?<span class="stars(.*?) starstop" title="力荐">[\s\S]+?.*?[\s\S]+?</span>[\s\S]+?<div class="power" style=".*?"></div>[\s\S]+?<span class="rating_per">(.*?)</span>[\s\S]+?<br />[\s\S]+?</div>'
        four_star_regex = '<div class="item">[\s\S]+?<span class="stars(.*?) starstop" title="推荐">[\s\S]+?.*?[\s\S]+?</span>[\s\S]+?<div class="power" style=".*?"></div>[\s\S]+?<span class="rating_per">(.*?)</span>[\s\S]+?<br />[\s\S]+?</div>'
        three_star_regex = '<div class="item">[\s\S]+?<span class="stars(.*?) starstop" title="还行">[\s\S]+?.*?[\s\S]+?</span>[\s\S]+?<div class="power" style=".*?"></div>[\s\S]+?<span class="rating_per">(.*?)</span>[\s\S]+?<br />[\s\S]+?</div>'
        two_star_regex = '<div class="item">[\s\S]+?<span class="stars(.*?) starstop" title="较差">[\s\S]+?.*?[\s\S]+?</span>[\s\S]+?<div class="power" style=".*?"></div>[\s\S]+?<span class="rating_per">(.*?)</span>[\s\S]+?<br />[\s\S]+?</div>'
        one_star_regex = '<div class="item">[\s\S]+?<span class="stars(.*?) starstop" title="很差">[\s\S]+?.*?[\s\S]+?</span>[\s\S]+?<div class="power" style=".*?"></div>[\s\S]+?<span class="rating_per">(.*?)</span>[\s\S]+?<br />[\s\S]+?</div>'

        # info1 = Spider().get_html(url)
        #
        # print(info1)
        info = Spider().get_info(url, fen=fen_regex,five_star=five_star_regex, four_star=four_star_regex,
                                 three_star=three_star_regex, two_star=two_star_regex,
                                 one_star=one_star_regex)
        # login_url = 'https://accounts.douban.com/passport/login?redir=https%3A%2F%2Fmovie.douban.com%2Fsubject%2F30163509%2F%3Ftag%3D%25E7%2583%25AD%25E9%2597%25A8%26from%3Dgaia_video'
        # info1 = Session().by_session_get_html(login_url, url, username='18234686259', password='guan950701.')
        # print(info1)
        # info = Session().get_html_info(login_url, url, username='18234686259', password='guan950701.',
        #                                five_star=five_star_regex, four_star=four_star_regex,
        #                                three_star=three_star_regex, two_star=two_star_regex,
        #                                one_star=one_star_regex
        #                                )
        try:
            info['fen'] = info['fen'][0]
            info['five_star'] = [{'name': i[0]+"星", 'fen': i[1]} for i in info['five_star']][0]
            info['four_star'] = [{'name': i[0] + "星", 'fen': i[1]} for i in info['four_star']][0]
            info['three_star'] = [{'name': i[0] + "星", 'fen': i[1]} for i in info['three_star']][0]
            info['two_star'] = [{'name': i[0] + "星", 'fen': i[1]} for i in info['two_star']][0]
            info['one_star'] = [{'name': i[0] + "星", 'fen': i[1]} for i in info['one_star']][0]
        except IndexError:
            pass

        print(info)
        return info



    #输入一部电影的链接，获取到电影的影评所有分页链接
    def get_movie_film_review_all_page_url(self, url):
        """
        输入一部电影的链接，获取到电影的影评
        :param url: https://movie.douban.com/subject/30163509/?tag=热门&from=gaia_video
        :return: [page_url,page_url,.........]
        """
        movie_id_regex = 'https://movie.douban.com/subject/(.*?)tag=.*?&from=gaia_video'

        movie_id = re.findall(movie_id_regex, url)[0]

        film_url = 'https://movie.douban.com/subject/{}/reviews'.format(movie_id[:-2])

        # print(film_url)

        max_page_regex = '<span class="thispage" data-total-page="(.*?)">1</span>'

        max_page = Spider().get_info(film_url, max_page=max_page_regex)

        # print(max_page)

        max_page_number = int(max_page['max_page'][0])

        # print(max_page_number)

        all_page_url = []

        for i in range(1,max_page_number+1):

            page_url = 'https://movie.douban.com/subject/{}/reviews?start={}'.format(movie_id, i*20-20)

            all_page_url.append(page_url)

        print(all_page_url)

        return all_page_url

    #输入一个电影影评的分页链接，获取到影评内容的链接
    def get_film_content_url(self, url):
        """
        输入一个电影影评的分页链接，获取到影评的链接
        :param url: https://movie.douban.com/subject/30163509/?/reviews?start=0
        :return:  {'content_url': ['https://movie.douban.com/review/9945766/',........]
        """
        content_url_regex = '<h2><a href="(.*?)">.*?</a></h2>'
        # content_url_id = '<div class="main review-item" id="(.*?)">'

        info = Spider().get_info(url, content_url=content_url_regex)

        print(info)

        # new_info = {}
        # new_info['content_url'] = ["https://movie.douban.com/j/review/{}/full".format(i) for i in info['content_url_id']]
        # print(new_info)

        return info

    #输入一评论内容的链接，获取到信息
    def get_movie_film_review(self, url):
        """
        输入一评论内容的链接，获取到信息
        :param url: https://movie.douban.com/j/review/9945766/full
        :return: {'author': '影探', 'time': '2019-02-07 12:53:45', 'content': ''}
        """
        author_regex = 'data-author="(.*?)"'
        time_regex = 'class="main-meta">(.*?)</span>'
        content_regex = 'data-original="1">(.*?)<div class="copyright">'

        info = Spider().get_html(url)

        print(info)

        info = Spider().get_info(url, author=author_regex, time=time_regex, content=content_regex)
        try:
            info['author'] = info['author'][0]
            info['time'] = info['time'][0]
            info['content'] = info['content'][0]
        except IndexError:
            pass
        print(info)



        # info = requests.get(url)
        # info.encoding = info.apparent_encoding
        # html = info.text
        # print(html)
        # author = '<p>作者：(.*?)<\/p>'
        # content_regex = 'data-original="1">(.*?)<div class="copyright">'
        # aa = re.findall(author, html)
        # print(aa)




    #————————————————功能————————————————
    #输入一个分类页面URL，把所有分类信息都保存在 class_movie 表
    def save_class_movie(self, url):

        info = self.get_all_class_url(url)

        for i in info:
            select_class_sql = 'SELECT * FROM class_movie where class_url="{}"'.format(i['class_name'])

            select_class = self.fetch(select_class_sql)

            if not select_class:

                insert_class_sql = 'INSERT INTO class_movie(class_name, class_url) VALUES (?,?)'

                self.insert(insert_class_sql, (i['class_name'], i['class_url']))

                print("添加{}类型成功".format(i['class_name']))




    #给class_queue添加任务，class_url
    def put_class_queue(self):
        select_class_sql = 'SELECT class_url FROM class_movie'

        new_cursor = self.conn.cursor()

        new_cursor.execute(select_class_sql)

        while True:

            take = new_cursor.fetchone()

            if take == None:

                self.class_queue.put(take)
                self.class_queue.put(take)
                self.class_queue.put(take)

                print("任务添加完毕")
                break

            self.class_queue.put(take)

    #输入一个分类界面URL，把此分页说有电影保存到movie表
    def save_one_class_movie(self, url):

        info = self.get_one_class_all_movie_url(url)

        select_class_sql = 'SELECT id FROM class_movie where class_url="{}"'.format(url)

        select_class = self.fetch(select_class_sql)

        print(select_class)

        class_id = select_class[0][0]

        for i in info:

            select_movie_sql = 'SELECT * FROM movie where movie_url="{}"'.format(i['movie_url'])

            select_movie = self.fetch(select_movie_sql)

            if not select_movie:

                insert_movie_sql = 'INSERT INTO movie(movie_name, movie_url, this_class_id) VALUES (?,?,?)'

                self.insert(insert_movie_sql, (i['movie_name'], i['movie_url'], class_id))

                print("电影{}添加成功".format(i['movie_name']))
            else:
                print("电影{}已经存在".format(i['movie_name']))

    #把全站电影保存到movie表
    def save_all_class_movie(self):

        while True:

            class_url = self.class_queue.get()

            if class_url == None:
                print("线程退出")
                self.class_queue.task_done()
                break

            active_num = threading.active_count()

            if active_num <= 4:

                threading.Thread(target=self.save_all_class_movie).start()

            self.save_one_class_movie(class_url)

    #启动添加movie表的线程
    def movie_thread(self):

        t = threading.Thread(target=self.put_class_queue)

        t.start()

        threading.Thread(target=self.save_all_class_movie).start()

        self.class_queue.join()




    #给movie_queue添加任务，movie_url,id
    def put_movie_queue(self):
        select_movie_sql = 'SELECT id,movie_url,movie_name FROM movie'

        new_cursor = self.conn.cursor()

        new_cursor.execute(select_movie_sql)

        while True:

            take = new_cursor.fetchone()

            if take == None:

                self.movie_queue.put(take)
                self.movie_queue.put(take)
                self.movie_queue.put(take)

                print("任务添加完毕")
                break

            self.movie_queue.put(take)
            print("添加一条任务")

    #保存电影信息到movie_info表
    def save_movie_info_db(self):

        print(2)

        while True:

            take = self.movie_queue.get()

            print(take)

            if take == None:
                print("任务结束")
                self.movie_queue.task_done()
                break

            active_num = threading.active_count()

            if active_num <= 4:
                threading.Thread(target=self.save_movie_info_db).start()

            movie_id = take[0]
            movie_url = take[1]
            movie_name = take[2]

            info = self.get_movie_info(movie_url)

            select_movie_sql = 'SELECT * FROM movie_info where this_movie_id={}'.format(movie_id)

            select_movie = self.fetch(select_movie_sql)

            if not select_movie:

                insert_movie_sql = 'INSERT INTO movie_info(driect, scirptwriiter, actor, genre_type, state, '\
                                   'language, data, run_time, IMDb, this_movie_id) values (?,?,?,?,?,?,?,?,?,?)'

                self.insert(insert_movie_sql, (str(info['driect']).replace("[", "").replace("]", ""),
                                               info['scirptwriter'],
                                               str(info['actor']).replace("[", "").replace("]", ""),
                                               info['genre_type'], info['state'], info['language'],
                                               info['data'], info['run_time'],
                                               str(info['IMDb']).replace("[", "").replace("]", ""),
                                               movie_id))
                print('{}信息添加完成'.format(movie_name))

            else:
                print('{}已经存在'.format(movie_name))

    #添加电影信息到movie_info表的线程
    def movie_info_thread(self):
        t = threading.Thread(target=self.put_movie_queue)

        t.start()

        print(3)

        threading.Thread(target=self.save_movie_info_db).start()

        self.movie_queue.join()




    #保存电影评分到grade表
    def save_film_grade_db(self):

        while True:

            take = self.movie_queue.get()
            print(take)
            if take == None:
                print("任务完成")
                self.movie_queue.task_done()
                break

            active_num = threading.active_count()
            if active_num <= 4:
                threading.Thread(target=self.save_film_grade_db).start()

            movie_id = take[0]
            movie_url = take[1]
            movie_name = take[2]
            print(22)
            info = self.get_movie_grade(movie_url)
            print(33)

            select_grade_sql = 'SELECT * FROM grade where this_movie_id={}'.format(movie_id)

            select_grade = self.fetch(select_grade_sql)

            if not select_grade:

                insert_grade_sql = 'INSERT INTO grade(five, four, three, two, one,' \
                                   'this_movie_id) VALUES (?,?,?,?,?,?)'

                self.insert(insert_grade_sql, (info['five_star']['fen'], info['four_star']['fen'],
                                               info['three_star']['fen'], info['two_star']['fen'],
                                               info['one_star']['fen'], movie_id))

                print("{}豆瓣评分添加完成".format(movie_name))

            else:

                print("{}评分已经存在".format(movie_name))

    #添加电影评分的线程
    def grade_thread(self):

        t = threading.Thread(target=self.put_movie_queue)

        t.start()
        print(1)

        threading.Thread(target=self.save_film_grade_db).start()

        self.movie_queue.join()





    #爬取一部电影所有的影评并写道数据库content表
    def save_all_film_review(self):
        """
        输入一部电影的链接返回所有的影评
        :param url:
        :return:
        """
        while True:

            take = self.movie_queue.get()
            print(take)
            if take == None:
                print("任务完成")
                self.movie_queue.task_done()
                break

            active_num = threading.active_count()
            if active_num <= 4:
                threading.Thread(target=self.save_all_film_review).start()

            movie_id = take[0]
            movie_url = take[1]
            movie_name = take[2]

            all_page_url = self.get_movie_film_review_all_page_url(movie_url)

            for page_url in all_page_url:

                all_content_url = self.get_film_content_url(page_url)

                for content_url in all_content_url['content_url']:

                    content_info = self.get_movie_film_review(content_url)

                    select_movie_sql = "SELECT * FROM content where this_movie_id={}".format(movie_id)

                    select_movie = self.fetch(select_movie_sql)

                    if not select_movie:

                        insert_content_sql = "INSERT INTO content(author, content, time, content_url, this_movie_id) VALUES (?,?,?,?,?)"

                        self.insert(insert_content_sql, (content_info['author'], content_info['content'],
                                                         content_info['time'], content_url, movie_id))

                        print("{}的一条影评添加成功".format(movie_name))

                    else:

                        print("影评已经存在")

    #添加影评内容的线程
    def content_thread(self):
        t = threading.Thread(target=self.put_movie_queue)

        t.start()
        print(1)

        threading.Thread(target=self.save_all_film_review).start()

        self.movie_queue.join()




    def my_thread(self):

        t = threading.Thread(target=self.save_class_movie)
        t.start()
        t.join()
        print("-----------------------class_movie 表 添加完毕-----------------------")

        t1 = threading.Thread(target=self.save_one_class_movie, args='https://movie.douban.com/explore#!type=movie&tag=经典&page_limit=20&page_start=0')
        t1.start()
        t.join()
        print("-----------------------movie 表 添加完毕-----------------------")

        t2 = threading.Thread(target=self.movie_info_thread)
        t2.start()
        t2.join()
        print("-----------------------movie_info 表 添加完毕-----------------------")

        t3 = threading.Thread(target=self.grade_thread)
        t3.start()
        t3.join()
        print("-----------------------grade 表 添加完毕-----------------------")

        t4 = threading.Thread(target=self.content_thread)
        t4.start()
        t4.join()
        print("-----------------------content 表 添加完毕-----------------------")





if __name__ == "__main__":

    x = DouBan(db_name='Movie.sqlite3')
    # x.get_movie_info(url='https://movie.douban.com/subject/27605698/?tag=%E7%83%AD%E9%97%A8&from=gaia_video')
    # x.get_all_class_url(url='https://movie.douban.com/explore#!type=movie&tag=热门&sort=recommend&page_limit=20&page_start=0')
    # x.get_one_class_all_movie_url(url='https://movie.douban.com/explore#!type=movie&tag=%E6%9C%80%E6%96%B0&page_limit=20&page_start=0')
    # x.get_movie_grade(url='https://movie.douban.com/subject/30163509/?tag=热门&from=gaia_video')
    # x.get_movie_film_review_all_page_url(url='https://movie.douban.com/subject/30163509/?tag=热门 &from=gaia_video')
    # x.get_film_content_url(url='https://movie.douban.com/subject/30163509/?tag=最新&from=gaia_video')
    x.get_movie_film_review(url='https://movie.douban.com/review/9945766/')



    # x.save_class_movie(url='https://movie.douban.com/explore#!type=movie&tag=热门&sort=recommend&page_limit=20&page_start=0')
    # x.save_one_class_movie(url='https://movie.douban.com/explore#!type=movie&tag=经典&page_limit=20&page_start=0')
    # x.movie_thread()
    # x.movie_info_thread()
    # x.grade_thread()
    # x.content_thread()
    # x.my_thread()